import pandas as pd
import numpy as np
from pathlib import Path

import plotly.express as px
pd.options.plotting.backend = 'plotly'

import plotly.io as pio
pio.renderers.default = "vscode"

from dsc80_utils import * # Feel free to uncomment and use this.

interactions = pd.read_csv('RAW_interactions.csv')
recipes = pd.read_csv('RAW_recipes.csv')

merged = recipes.merge(interactions, how='left', left_on = 'id', right_on='recipe_id')
merged['rating'] = merged['rating'].replace(0, np.nan)

avg_rating = merged.groupby('id')['rating'].mean()

data = merged.set_index('id')
data['avg_rating'] = avg_rating

# drop duplicate columns
data = data.drop(columns=['recipe_id'])

data

data.shape

(234429, 16)

data.dtypes

name               object
minutes             int64
contributor_id      int64
                   ...   
rating            float64
review             object
avg_rating        float64
Length: 16, dtype: object

data.describe()

nutrition = data["nutrition"].str.strip("[]").str.split(",", expand=True).astype(float)
nutrition.columns = ["calories","total_fat","sugar","sodium","protein","saturated_fat","carbs"]
data = pd.concat([data, nutrition], axis=1)

print(f'Updated dataset shape: {data.shape}')

Updated dataset shape: (234429, 23)

fig = px.histogram(
    data_frame=data,
    x='avg_rating',
    nbins=30,    
    title='Distribution of Average Ratings',
    histnorm='probability'
)

fig.update_layout(
    xaxis_title="Average Rating",
    yaxis_title="Count",
)
fig.show()

fig = px.histogram(
    data_frame=data,
    x='minutes',   
    title='Distribution of Duration (Minutes)',
    histnorm='probability',
)

fig.update_layout(
    xaxis_title="Duration (Minutes)",
    yaxis_title="Count",
)

fig.update_traces(
    xbins=dict(
        size=1000
    )
)

fig.show()

fig = px.histogram(
    data_frame=data[data['minutes'] < 1000],
    x='minutes',   
    title='Distribution of Duration (Minutes)',
    histnorm='probability'
)

fig.update_layout(
    xaxis_title="Duration (Minutes)",
    yaxis_title="Count",
)

fig.update_traces(
    xbins=dict(
        size=10
    )
)

fig.show()

data = data[data['minutes'] < 400]
fig = px.histogram(
    data_frame=data,
    x='minutes',   
    title='Distribution of Duration (Minutes)',
    histnorm='probability',
)

fig.update_layout(
    xaxis_title="Duration (Minutes)",
    yaxis_title="Count",
)

fig.update_traces(
    xbins=dict(
        size=10
    )
)
fig.show()

print(f'Updated dataset shape: {data.shape}')

Updated dataset shape: (228374, 23)

fig = px.histogram(
    data_frame=data,
    x='calories',   
    title='Distribution of Calories',
    histnorm='probability',
)

fig.update_layout(
    xaxis_title="Calories",
    yaxis_title="Count",
)

fig.update_traces(
    xbins=dict(
        size=1000
    )
)

fig.show()

data = data[(data['calories'] < 2000) & (data['calories'] >= 0)]

fig = px.histogram(
    data_frame=data,
    x='calories',   
    title='Distribution of Calories',
    histnorm='probability',
)

fig.update_layout(
    xaxis_title="Calories",
    yaxis_title="Count",
)

fig.update_traces(
    xbins=dict(
        size=100
    )
)

fig.show()

print(f'Updated dataset shape: {data.shape}')

Updated dataset shape: (224755, 23)

data[['total_fat', 'sugar', 'sodium', 'protein', 'saturated_fat', 'carbs']].describe()

data = (data[
    (data['total_fat'] <= 100) &
    (data['sugar'] <= 100) &
    (data['sodium'] <= 100) &
    (data['protein'] <= 100) &
    (data['saturated_fat'] <= 100) &
    (data['carbs'] <= 100)
])

data[['total_fat', 'sugar', 'sodium', 'protein', 'saturated_fat', 'carbs']].describe()

print(f'Updated dataset shape: {data.shape}')

Updated dataset shape: (173222, 23)

# define stop words that we want to avoid
stop_words = [
    "the", "a", "an", "and", "or", "with", "without", "from", "of", 
    "for", "to", "in", "on", "by", "s", 
    "this", "that", "these", "those", "it", "its", "is", "are", "was", "were", 
    "be", "been", "being", "at", "as", "but", "if", "then", "so", "too", 
    "can", "could", "may", "might", "will", "would", "shall", "should", 
    "i", "you", "he", "she", "we", "they", "me", "him", "her", "us", "them", 
    "my", "your", "his", "her", "our", "their", "mine", "yours", "hers", 
    "ours", "theirs", "no", "not", "do", "does", "did", "done", "have", "has", "had", 
    "up", "down", "over", "under", "again", "once", "here", "there", "when", "where", "why", "how", "like", 
    "use", "very", "also", "great", "also", "great"
]

# analyze the most repeated words in name:
word_cleaner = lambda x: x.lower().strip()

# filter out all words present in the recipe names 
all_split_names = data['name'].apply(lambda x: str(x).split())
all_unigrams_in_names = (
    pd.Series([word_cleaner(word) for name in all_split_names for word in name if word not in stop_words])
)
all_unigrams_in_names

0                  1
1           brownies
2              world
             ...    
718872         sugar
718873    shortbread
718874       cookies
Length: 718875, dtype: object

# top 10 most common unigrams 
unigram_counts = all_unigrams_in_names.value_counts()
top_ten_name_unigrams = unigram_counts.sort_values().nlargest(n = 10)

fig = px.bar(top_ten_name_unigrams, title='Top 10 Most Frequent Unigrams in Recipe Names')

fig.update_layout(
    xaxis_title="Unigrams",
    yaxis_title="Count"
)

fig.show()

# analyze the most repeated words in name:
word_cleaner = lambda x: x.lower().strip()

# filter out all words present in the recipe names 
all_bigrams = []
for i in range(len(all_unigrams_in_names) - 1):
    all_bigrams.append((all_unigrams_in_names[i], all_unigrams_in_names[i + 1]))

all_bigrams = pd.Series(all_bigrams)

# top 10 most common bigrams
bigram_counts = all_bigrams.value_counts()
top_ten_name_bigrams = bigram_counts.nlargest(n = 10)  # already sorted descending

bigram_df = top_ten_name_bigrams.reset_index()
bigram_df.columns = ['bigram', 'count']
bigram_df['bigram_str'] = bigram_df['bigram'].apply(lambda x: f"{x[0]} {x[1]}")

fig = px.bar(
    bigram_df,
    x='bigram_str',
    y='count',
    title="Top 10 Most Frequent Bigrams in Recipe Names",
    labels={'bigram_str':'Bigram', 'count':'Count'}
)

fig.show()

# analyze the most repeated words in name:
tag_cleaner = lambda x: x.replace('\'', '').replace('[', '').replace(']', '').replace(',', '')

cleaned_tags = data['tags'].apply(tag_cleaner)
final_tags = pd.Series([word for tag in cleaned_tags for word in tag.split()])

tag_counts = final_tags.value_counts()
top_ten_tags = tag_counts.sort_values().nlargest(n = 10)

fig = px.bar(top_ten_tags, title='Top 10 Most Frequent Tags')

fig.update_layout(
    xaxis_title="Tags",
    yaxis_title="Count"
)

fig.show()

# analyze the most repeated words in name:
steps_cleaner = lambda x: str(x).replace('\'', '').replace('[', '').replace(']', '').replace(',', '')

cleaned_ingredients = data['ingredients'].apply(steps_cleaner)
cleaned_descriptions_ingredients = (
    pd.Series([word for step in cleaned_ingredients for word in step.split() if word not in stop_words])
)

ingredients_counts = cleaned_descriptions_ingredients.value_counts()
top_ten_ingredients = ingredients_counts.sort_values().nlargest(n = 10)

fig = px.bar(top_ten_ingredients, title='Top 10 Most Frequent Ingredients')

fig.update_layout(
    xaxis_title="Ingredients",
    yaxis_title="Count"
)

fig.show()

# count recipes submitted per year and plot a line chart
submitted_dt = pd.to_datetime(recipes['submitted'], errors='coerce')
year_counts = (submitted_dt.dt.year
               .value_counts()
               .sort_index()
               .rename_axis('year')
               .reset_index(name='count'))

fig = px.line(year_counts, x='year', y='count', markers=True,
              title='Number of Recipes Submitted per Year')
fig.update_layout(xaxis_title='Year', yaxis_title='Number of Recipes')
fig.show()

df_plot = data[['rating', 'calories']]
fig = px.box(
    df_plot,
    x='rating',
    y='calories',
    title='Calories by Rating (1-5)',
    labels={'rating': 'Rating', 'calories': 'Calories'},
    points='outliers'
)
fig.update_layout(xaxis=dict(categoryorder='array',
                             categoryarray=['1','2','3','4','5']))
fig.show()

df_plot = data[['rating', 'n_steps']]
fig = px.box(
    df_plot,
    x='rating',
    y='n_steps',
    title='Number of Steps by Rating (1-5)',
    labels={'rating': 'Rating', 'n_steps': 'Number of Steps'},
    points='outliers'
)

fig.update_layout(xaxis=dict(categoryorder='array',
                             categoryarray=['1','2','3','4','5']))
fig.show()

df_plot = data[['rating', 'n_ingredients']]
fig = px.box(
    df_plot,
    x='rating',
    y='n_ingredients',
    title='Number of Ingredients by Rating (1-5)',
    labels={'rating': 'Rating', 'n_ingredients': 'Number of Ingredients'},
    points='outliers'
)
fig.update_layout(xaxis=dict(categoryorder='array',
                             categoryarray=['1','2','3','4','5']))
fig.show()

# select numeric columns only
numeric_features = data[['minutes', 'n_steps', 'n_ingredients']]

# create correlation matrix and plot with plotly express
corr = numeric_features.corr()

fig = px.imshow(
    corr,
    text_auto=True,
    color_continuous_scale='temps_r',
    zmin=-1, zmax=1,
    labels=dict(x='', y='', color='Correlation'),
    x=corr.columns,
    y=corr.index,
    title='Correlation Between Recipe Features'
)

fig.update_layout(width=600, height=500)
fig.show()

fig = px.scatter_matrix(
    data[['calories', 'total_fat', 'sugar', 'sodium', 'protein', 'saturated_fat', 'carbs']],
    dimensions=['calories', 'total_fat', 'sugar', 'sodium', 'protein', 'saturated_fat', 'carbs'],
    title='Pairplot of Recipe Features',
    labels=(
        {col: col.replace('_', ' ').title() for col in ['calories', 'total_fat', 'sugar', 'sodium', 'protein', 'saturated_fat', 'carbs']}
    )
)

fig.update_layout(width=1200, height=900)
fig.show()

# plot ingredients and steps, stratified by rating 
df_scatter = data[['n_ingredients', 'n_steps', 'rating']].dropna()
df_scatter['rating_str'] = df_scatter['rating'].astype(str)

fig = px.scatter(
    df_scatter,
    x='n_ingredients',
    y='n_steps',
    color='rating_str',
    title='Number of Ingredients vs Number of Steps (stratified by Rating)',
    labels={'n_ingredients': 'Number of Ingredients', 'n_steps': 'Number of Steps', 'rating_str': 'Rating'},
    opacity=0.6,
    hover_data=['rating'],
    category_orders={
    'rating_str': ['5.0', '4.0', '3.0', '2.0', '1.0']
}
)

fig.update_traces(marker=dict(size=6))
fig.show()

data.isna().sum()

name              0
minutes           0
contributor_id    0
                 ..
protein           0
saturated_fat     0
carbs             0
Length: 23, dtype: int64

data = data.dropna(subset=['description', 'review'])

# 1. Create missing indicator
data['avg_rating_missing'] = data['avg_rating'].isna().astype(int)

df = data[['minutes', 'avg_rating_missing']]

missing_group = df[df['avg_rating_missing'] == 1]['minutes']
present_group = df[df['avg_rating_missing'] == 0]['minutes']

# 2. Observed statistic
observed_stat = abs(missing_group.mean() - present_group.mean())

# 3. Permutation test
num_permutations = 1000
np.random.seed(42)

feature_values = df['minutes'].values
missingness = df['avg_rating_missing'].values

permuted_stats = []

for _ in range(num_permutations):
    shuffled = np.random.permutation(feature_values)
    perm_missing = shuffled[missingness == 1]
    perm_present = shuffled[missingness == 0]
    permuted_stats.append(abs(perm_missing.mean() - perm_present.mean()))

permuted_stats = np.array(permuted_stats)

# 4. p-value
p_value = (permuted_stats >= observed_stat).mean()

print(f"\nObserved difference in mean minutes: {observed_stat:.4f}")
print(f"P-value: {p_value:.4f}")

# Plot permutation distribution
fig = px.histogram(
    x=permuted_stats,
    nbins=50,
    title="Permutation Distribution of Test Statistic",
    labels={'x': 'Abs Difference in Mean Minutes', 'y': 'Frequency'}
)

fig.add_vline(x=observed_stat, line_width=3, line_color="red", annotation_text=f"Observed = {observed_stat:.4f}")
fig.update_layout(xaxis_title='Abs Difference in Mean Minutes', yaxis_title='Frequency')
fig.show()

Observed difference in mean minutes: 10.6094
P-value: 0.0000

# Impute missing avg_rating with mean rating per minutes quantile

# Create quantile bins for minutes
data['minutes_quartile'] = pd.qcut(data['minutes'], q=4, labels=['Q1', 'Q2', 'Q3', 'Q4'], duplicates='drop')

# Calculate mean avg_rating for each quartile (excluding missing values)
quartile_means = data.groupby('minutes_quartile', observed=True)['avg_rating'].transform('mean')

# Impute missing avg_rating values with quartile means
print("Missing values before imputation:", data['avg_rating'].isna().sum())
data['avg_rating'] = data['avg_rating'].fillna(quartile_means)

# Verify imputation
print("Missing values after imputation:", data['avg_rating'].isna().sum())

# Show mean ratings by quartile
print("\nMean avg_rating by minutes quartile:")
print(data.groupby('minutes_quartile', observed=True)['avg_rating'].mean())

Missing values before imputation: 1839
Missing values after imputation: 0

Mean avg_rating by minutes quartile:
minutes_quartile
Q1    4.71
Q2    4.68
Q3    4.67
Q4    4.66
Name: avg_rating, dtype: float64

# drop intermediate columns used for imputation & individual recipe ratings
data = data.drop(columns=['minutes_quartile', 'avg_rating_missing', 'rating'])

print(f'Final dataset shape after cleaning, filtering, and imputation: {data.shape}')

Final dataset shape after cleaning, filtering, and imputation: (173088, 22)

# confirm no missing values remain
data.isna().sum().sum()

np.int64(0)

from scipy.stats import ks_2samp

high = data.loc[data["protein"] > 16, "calories"]#.sort_values(ascending=False)
low = data.loc[data["protein"] <= 16, "calories"]#.sort_values(ascending=False)
stat, p = ks_2samp(high, low)
conclusion = "reject H0" if p < 0.05 else "accept H0"
print(f"p = {p}", conclusion)

p = 0.0 reject H0

def ecdf(data):
    data_sorted = np.sort(data)
    yvals = np.arange(1, len(data) + 1) / len(data)
    return data_sorted, yvals

# Calculate ECDFs
high_sort, high_ecdf = ecdf(high)
low_sort, low_ecdf = ecdf(low)

# Create DataFrame for plotting
df_high = pd.DataFrame({'calories': high_sort, 'ecdf': high_ecdf, 'group': 'High Protein'})
df_low = pd.DataFrame({'calories': low_sort, 'ecdf': low_ecdf, 'group': 'Low Protein'})
df_combined = pd.concat([df_high, df_low])

# Plot ECDFs
fig = px.line(
    df_combined,
    x='calories',
    y='ecdf',
    color='group',
    title='K-S Visualization',
    labels={'calories': 'Calories', 'ecdf': 'ECDF'},
    line_shape='hv'
)

# Calculate KS distance line
all_points = np.sort(np.concatenate([high, low]))
high_cdf = np.searchsorted(high_sort, all_points, side='right') / len(high)
low_cdf = np.searchsorted(low_sort, all_points, side='right') / len(low)
idx_max = np.argmax(np.abs(high_cdf - low_cdf))

# Add vertical line for KS distance
fig.add_scatter(
    x=[all_points[idx_max], all_points[idx_max]],
    y=[min(high_cdf[idx_max], low_cdf[idx_max]), max(high_cdf[idx_max], low_cdf[idx_max])],
    mode='lines',
    line=dict(color='red', width=2),
    name=f'KS distance D = {stat:.3f}'
)

fig.update_layout(width=700, height=500)
fig.show()

from collections import Counter
import ast

tag_lst_lst = data["tags"].apply(ast.literal_eval).to_list()
tags_cnt = sum([Counter(tag_lst) for tag_lst in tag_lst_lst], Counter())

import re
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import Pipeline
from sklearn.metrics import make_scorer, mean_squared_error

def find_tag(tag_lst, tag):
    return 1 if len(re.findall(fr"'{tag}'", tag_lst)) != 0 else 0

all_tags = ["preparation", "dietary", "easy", "occasion", "main-dish", "vegetables", "meat"]
numeric_features = ["n_steps","n_ingredients"]
one_hot_features = []
data_pred = data.copy()

for tag in all_tags:
    data_pred[f"with_{tag}"] = data_pred["tags"].apply(find_tag, args = (tag,))
    pred_tag = "with_" + tag
    one_hot_features.append(pred_tag)

model = Pipeline([
    ("regressor", LinearRegression())
])

x_var = data_pred[numeric_features + one_hot_features]
y_var = data_pred["minutes"]
X_train, X_test, y_train, y_test = train_test_split(
    x_var, y_var, test_size=0.2, random_state=42
)

#K-fold validation with evaluation
rmse_scorer = make_scorer(mean_squared_error, squared = False)
cv_scores = cross_val_score(model, X_train, y_train, cv = 5, scoring=rmse_scorer)


model.fit(X_train, y_train)
y_pred = model.predict(X_test)
rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print("RMSE:", rmse)

RMSE: 53.045515437919754

c:\.local\share\mamba\envs\dsc80\Lib\site-packages\sklearn\metrics\_regression.py:492: FutureWarning:

'squared' is deprecated in version 1.4 and will be removed in 1.6. To calculate the root mean squared error, use the function'root_mean_squared_error'.

c:\.local\share\mamba\envs\dsc80\Lib\site-packages\sklearn\metrics\_regression.py:492: FutureWarning:

'squared' is deprecated in version 1.4 and will be removed in 1.6. To calculate the root mean squared error, use the function'root_mean_squared_error'.

c:\.local\share\mamba\envs\dsc80\Lib\site-packages\sklearn\metrics\_regression.py:492: FutureWarning:

'squared' is deprecated in version 1.4 and will be removed in 1.6. To calculate the root mean squared error, use the function'root_mean_squared_error'.

c:\.local\share\mamba\envs\dsc80\Lib\site-packages\sklearn\metrics\_regression.py:492: FutureWarning:

'squared' is deprecated in version 1.4 and will be removed in 1.6. To calculate the root mean squared error, use the function'root_mean_squared_error'.

c:\.local\share\mamba\envs\dsc80\Lib\site-packages\sklearn\metrics\_regression.py:492: FutureWarning:

'squared' is deprecated in version 1.4 and will be removed in 1.6. To calculate the root mean squared error, use the function'root_mean_squared_error'.

import re
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import Pipeline
from sklearn.metrics import make_scorer, mean_squared_error
from sklearn.preprocessing import FunctionTransformer, PolynomialFeatures
from sklearn.compose import ColumnTransformer


def find_tag(tag_lst, tag):
    return 1 if len(re.findall(fr"'{tag}'", tag_lst)) != 0 else 0


all_features = numeric_features + one_hot_features

log_transformer = FunctionTransformer(np.log1p, validate=False)

def manual_features(X):
    steps = X.iloc[:, 0]
    ings = X.iloc[:, 1]
    interaction = steps * ings
    ratio = steps / (1 + ings)
    return np.vstack([interaction, ratio]).T

manual_feature_names = ["steps_x_ings", "steps_div_ings"]
manual_feature_transformer = FunctionTransformer(manual_features, validate=False)
poly_interact = PolynomialFeatures(
    degree=2,
    interaction_only=False,
    include_bias=False
)


preprocessor = ColumnTransformer(
    transformers=[
        ("log_numeric", log_transformer, numeric_features),
        ("manual_feat", manual_feature_transformer, numeric_features),
        ("poly_interaction", poly_interact, all_features)
    ],
    remainder="passthrough"
)

model = Pipeline([
    ("preprocess", preprocessor),
    ("regressor", LinearRegression())
])


model.fit(X_train, y_train)
y_pred = model.predict(X_test)
rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print("RMSE:", rmse)

RMSE: 52.23256606605837

from sklearn.preprocessing import PolynomialFeatures

poly_preprocessor = ColumnTransformer(
    transformers=[
        ("poly_interaction", poly_interact, all_features)
    ],
    remainder="passthrough"
)

poly_model = Pipeline([
    ("pre_combine", poly_preprocessor),
    ("poly", PolynomialFeatures(degree = 2, include_bias = True)),
    ("linear", LinearRegression())
])


poly_model.fit(X_train, y_train)
y_pred = poly_model.predict(X_test)
rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print("RMSE:", rmse)

RMSE: 51.23243046325278

linear_step = poly_model.named_steps['linear']
if hasattr(linear_step, 'coef_'):
    coefs = linear_step.coef_
    poly_features = poly_model.named_steps['poly'].get_feature_names_out(
        poly_model.named_steps['pre_combine'].get_feature_names_out()
    )
    coefs = poly_model.named_steps['linear'].coef_
    feature_importance = pd.Series(coefs, index = poly_features)
    important_features = feature_importance[feature_importance.abs() > 0.01].sort_values(ascending=False)
    print("features that matter:")
    print(important_features.iloc[:10])

features that matter:
poly_interaction__with_vegetables                                    169.73
poly_interaction__with_vegetables^2                                  169.73
poly_interaction__with_vegetables^2                                  169.73
                                                                      ...  
poly_interaction__with_occasion poly_interaction__with_occasion^2     78.30
poly_interaction__with_occasion^2                                     78.30
poly_interaction__with_occasion^2                                     78.30
Length: 10, dtype: float64

from sklearn.ensemble import RandomForestRegressor

rf_model = Pipeline([
    ("preprocess", poly_preprocessor),
    ("rf", RandomForestRegressor(n_estimators = 200, max_depth = 10, random_state = 42, n_jobs = -1))
])

rf_model.fit(X_train, y_train)
y_pred_rf = rf_model.predict(X_test)
rmse_rf = np.sqrt(mean_squared_error(y_test, y_pred_rf))
print("\nRandom Forest RMSE:", rmse_rf)

Random Forest RMSE: 47.98761903038597

df_eval = X_test.copy()
df_eval['y_true'] = y_test
df_eval['y_pred'] = y_pred_rf
df_eval['error'] = df_eval['y_true'] - df_eval['y_pred']
df_eval['sq_error'] = df_eval['error'] ** 2

meat = df_eval[df_eval['with_meat'] == 1]
vegetable = df_eval[df_eval['with_vegetables'] == 1]

print(f"Meat-based recipes evaluation: {np.sqrt(meat['sq_error'].mean())}")
print(f"Vegetable-based recipes evaluation: {np.sqrt(vegetable['sq_error'].mean())}")

Meat-based recipes evaluation: 66.2992687694115
Vegetable-based recipes evaluation: 44.755841002768996

# Extract squared errors
sq_error_meat = meat['sq_error'].values
sq_error_veg = vegetable['sq_error'].values

# Observed RMSE difference
observed_diff = np.sqrt(sq_error_meat.mean()) - np.sqrt(sq_error_veg.mean())

# Permutation test
combined = np.concatenate([sq_error_meat, sq_error_veg])
n_perm = 5000
count = 0

test_stats = []
for _ in range(n_perm):
    np.random.shuffle(combined)
    perm_meat = combined[:len(sq_error_meat)]
    perm_veg = combined[len(sq_error_meat):]
    
    perm_diff = np.sqrt(perm_meat.mean()) - np.sqrt(perm_veg.mean())
    
    test_stats.append(perm_diff)

p_value = sum(abs(stat) >= abs(observed_diff) for stat in test_stats) / n_perm

print("Observed RMSE difference:", observed_diff)
print("Permutation test p-value:", p_value)

Observed RMSE difference: 21.543427766642502
Permutation test p-value: 0.0

	minutes	contributor_id	n_steps	n_ingredients	user_id	rating	avg_rating
count	2.34e+05	2.34e+05	234429.00	234429.00	2.34e+05	219393.00	231652.00
mean	1.07e+02	1.24e+07	10.02	9.07	2.22e+08	4.68	4.68
std	3.29e+03	1.48e+08	6.44	3.82	6.19e+08	0.71	0.50
...	...	...	...	...	...	...	...
50%	3.50e+01	4.47e+05	9.00	9.00	4.97e+05	5.00	4.86
75%	6.00e+01	7.77e+05	13.00	11.00	1.30e+06	5.00	5.00
max	1.05e+06	2.00e+09	100.00	37.00	2.00e+09	5.00	5.00

	total_fat	sugar	sodium	protein	saturated_fat	carbs
count	224755.00	224755.00	224755.00	224755.00	224755.00	224755.00
mean	27.86	50.56	26.49	30.66	34.60	11.18
std	30.24	90.07	85.23	34.24	42.51	12.13
...	...	...	...	...	...	...
50%	19.00	21.00	14.00	17.00	21.00	8.00
75%	37.00	55.00	32.00	47.00	48.00	15.00
max	340.00	1925.00	4727.00	516.00	648.00	165.00

Analysis of Food.com Recipes and Reviews¶

Step 1: Introduction¶

Step 2: Data Cleaning and Exploratory Data Analysis¶

*also based on the fact that the reccommended daily calorie intake for a typical adult is around 2000 calories.¶

Step 3: Assessment of Missingness¶

Step 4: Hypothesis Testing¶

Step 5: Framing a Prediction Problem¶

Step 6: Baseline Model¶

Step 7: Final Model¶

Step 8: Fairness Analysis¶

	name	minutes	contributor_id	submitted	...	date	rating	review	avg_rating
id
333281	1 brownies in the world best ever	40	985201	2008-10-27	...	2008-11-19	4.0	These were pretty good, but took forever to ba...	4.0
453467	1 in canada chocolate chip cookies	45	1848091	2011-04-11	...	2012-01-26	5.0	Originally I was gonna cut the recipe in half ...	5.0
306168	412 broccoli casserole	40	50969	2008-05-30	...	2008-12-31	5.0	This was one of the best broccoli casseroles t...	5.0
...	...	...	...	...	...	...	...	...	...
298509	cookies by design sugar shortbread cookies	20	506822	2008-04-15	...	2008-06-19	1.0	This recipe tastes nothing like the Cookies by...	3.0
298509	cookies by design sugar shortbread cookies	20	506822	2008-04-15	...	2010-02-08	5.0	yummy cookies, i love this recipe me and my sm...	3.0
298509	cookies by design sugar shortbread cookies	20	506822	2008-04-15	...	2014-11-01	NaN	I work at a Cookies By Design and can say this...	3.0